
library(tidyverse)
library(dbplyr)
library(fst)
library(data.table)
library(tictoc)
#library(DBI)

path <- "/jbod/projects/Credit_Scoring_ML_MersaultMoultonSantucci/data_qtr_rand_no_cid_before_consumer_group_var/"
# path_scratch <- "/scratch/projects/Meursault_credit_scoring_ml/"

# This line opens up a connection with the server
# Function arguments removed for privacy reasons
conn <- DBI::dbConnect()


DBI::dbListObjects(conn)

v_qtr_to_consider <- c(
  "2000-03-01", "2000-06-01", "2000-09-01", "2000-12-01",
  "2001-03-01", "2001-06-01", "2001-09-01", "2001-12-01",
  "2002-03-01", "2002-06-01", "2002-09-01", "2002-12-01",
  "2003-03-01", "2003-06-01", "2003-09-01", "2003-12-01",
  "2004-03-01", "2004-06-01", "2004-09-01", "2004-12-01",
  "2005-03-01", "2005-06-01", "2005-09-01", "2005-12-01",
  "2006-03-01", "2006-06-01", "2006-09-01", "2006-12-01",
  "2007-03-01", "2007-06-01", "2007-09-01", "2007-12-01",
  "2008-03-01", "2008-06-01", "2008-09-01", "2008-12-01",
  "2009-03-01", "2009-06-01", "2009-09-01", "2009-12-01",
  "2010-03-01", "2010-06-01", "2010-09-01", "2010-12-01",
  "2011-03-01", "2011-06-01", "2011-09-01", "2011-12-01",
  "2012-03-01", "2012-06-01", "2012-09-01", "2012-12-01",
  "2013-03-01", "2013-06-01", "2013-09-01", "2013-12-01",
  "2014-03-01", "2014-06-01", "2014-09-01", "2014-12-01",
  "2015-03-01", "2015-06-01", "2015-09-01", "2015-12-01",
  "2016-03-01", "2016-06-01", "2016-09-01", "2016-12-01",
  "2017-03-01", "2017-06-01", "2017-09-01", "2017-12-01",
  "2018-03-01", "2018-06-01", "2018-09-01", "2018-12-01",
  "2019-03-01", "2019-06-01", "2019-09-01", "2019-12-01",
  "2020-03-01", "2020-06-01", "2020-09-01", "2020-12-01",
  "2021-03-01", "2021-06-01")



tbl_consumer <- tbl()
tbl_consumer_dynamic <- tbl()
tbl_credit_trends_attr <- tbl()
tbl_consumer_modeling_attr <- tbl()
tbl_consumer_attr <- tbl()


v_rand_no_cid <- 0:99


for (i_qtr in v_qtr_to_consider) {
  qtr_ <- i_qtr
  for (rand_no_cid_ in v_rand_no_cid) {
    
    
    tic()
    tbl_consumer_sample <- tbl_consumer %>% 
      filter(rand_no_cid == rand_no_cid_) %>% 
      select(cid)
    
    df_qtr <- tbl_consumer_dynamic %>% 
      inner_join(tbl_consumer_sample, copy = TRUE) %>% 
      filter(qtr == qtr_) %>% 
      filter(dup_cid_flag_d == 1) %>% 
      inner_join(tbl_credit_trends_attr, copy = TRUE) %>% 
      inner_join(tbl_consumer_modeling_attr, copy = TRUE) %>% 
      inner_join(tbl_consumer_attr, copy = TRUE) %>% 
      collect(n = Inf) 
    
    
    df_qtr_unique <- df_qtr %>% 
      data.table() %>% 
      unique(by = c("cid", "qtr")) %>% 
      as_tibble()
    
    qtr_string <- str_replace_all(qtr_, "-", "") 
    rand_no_cid_string <- str_pad(rand_no_cid_, 4, "left", pad = "0")
    if(!dir.exists(path)){ dir.create(path, recursive = TRUE)}
    
    file_name_out <- paste0(path, "ccp_", qtr_string, "_", rand_no_cid_string , ".fst")
    write_fst(df_qtr_unique, file_name_out)
    
    print(paste0(qtr_, " ", rand_no_cid_))
    print(i_qtr)
    toc()
  }
}



